{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Use Advantage Actor-Critic to Play Acrobot-v1\n",
    "\n",
    "TensorFlow version"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "\n",
    "import sys\n",
    "import logging\n",
    "import itertools\n",
    "\n",
    "import numpy as np\n",
    "np.random.seed(0)\n",
    "import pandas as pd\n",
    "import gym\n",
    "import matplotlib.pyplot as plt\n",
    "import tensorflow.compat.v2 as tf\n",
    "tf.random.set_seed(0)\n",
    "from tensorflow import keras\n",
    "from tensorflow import nn\n",
    "from tensorflow import optimizers\n",
    "from tensorflow import losses\n",
    "from tensorflow.keras import layers\n",
    "from tensorflow.keras import models\n",
    "\n",
    "logging.basicConfig(level=logging.DEBUG,\n",
    "        format='%(asctime)s [%(levelname)s] %(message)s',\n",
    "        stream=sys.stdout, datefmt='%H:%M:%S')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "15:40:00 [INFO] env: <AcrobotEnv<Acrobot-v1>>\n",
      "15:40:00 [INFO] action_space: Discrete(3)\n",
      "15:40:00 [INFO] observation_space: Box(-28.274333953857422, 28.274333953857422, (6,), float32)\n",
      "15:40:00 [INFO] reward_range: (-inf, inf)\n",
      "15:40:00 [INFO] metadata: {'render.modes': ['human', 'rgb_array'], 'video.frames_per_second': 15}\n",
      "15:40:00 [INFO] _max_episode_steps: 500\n",
      "15:40:00 [INFO] _elapsed_steps: None\n",
      "15:40:00 [INFO] id: Acrobot-v1\n",
      "15:40:00 [INFO] entry_point: gym.envs.classic_control:AcrobotEnv\n",
      "15:40:00 [INFO] reward_threshold: -100.0\n",
      "15:40:00 [INFO] nondeterministic: False\n",
      "15:40:00 [INFO] max_episode_steps: 500\n",
      "15:40:00 [INFO] _kwargs: {}\n",
      "15:40:00 [INFO] _env_name: Acrobot\n"
     ]
    }
   ],
   "source": [
    "env = gym.make('Acrobot-v1')\n",
    "env.seed(0)\n",
    "for key in vars(env):\n",
    "    logging.info('%s: %s', key, vars(env)[key])\n",
    "for key in vars(env.spec):\n",
    "    logging.info('%s: %s', key, vars(env.spec)[key])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "class AdvantageActorCriticAgent:\n",
    "    def __init__(self, env):\n",
    "        self.action_n = env.action_space.n\n",
    "        self.gamma = 0.99\n",
    "\n",
    "        self.actor_net = self.build_net(hidden_sizes=[100,],\n",
    "                output_size=self.action_n, output_activation=nn.softmax,\n",
    "                loss=losses.categorical_crossentropy,\n",
    "                learning_rate=0.0001)\n",
    "        self.critic_net = self.build_net(hidden_sizes=[100,],\n",
    "                learning_rate=0.0002)\n",
    "\n",
    "    def build_net(self, hidden_sizes, output_size=1,\n",
    "                activation=nn.relu, output_activation=None,\n",
    "                loss=losses.mse, learning_rate=0.001):\n",
    "        model = keras.Sequential()\n",
    "        for hidden_size in hidden_sizes:\n",
    "            model.add(layers.Dense(units=hidden_size,\n",
    "                    activation=activation))\n",
    "        model.add(layers.Dense(units=output_size,\n",
    "                activation=output_activation))\n",
    "        optimizer = optimizers.Adam(learning_rate)\n",
    "        model.compile(optimizer=optimizer, loss=loss)\n",
    "        return model\n",
    "\n",
    "    def reset(self, mode=None):\n",
    "        self.mode = mode\n",
    "        if self.mode == 'train':\n",
    "            self.trajectory = []\n",
    "            self.discount = 1.\n",
    "\n",
    "    def step(self, observation, reward, done):\n",
    "        probs = self.actor_net.predict(observation[np.newaxis])[0]\n",
    "        action = np.random.choice(self.action_n, p=probs)\n",
    "        if self.mode == 'train':\n",
    "            self.trajectory += [observation, reward, done, action]\n",
    "            if len(self.trajectory) >= 8:\n",
    "                self.learn()\n",
    "            self.discount *= self.gamma\n",
    "        return action\n",
    "\n",
    "    def close(self):\n",
    "        pass\n",
    "\n",
    "    def learn(self):\n",
    "        state, _, _, action, next_state, reward, done, _ \\\n",
    "                = self.trajectory[-8:]\n",
    "        states = state[np.newaxis]\n",
    "        v = self.critic_net.predict(states)\n",
    "        next_v = self.critic_net.predict(next_state[np.newaxis])\n",
    "        target = reward + (1. - done) * self.gamma * next_v\n",
    "        td_error = target - v\n",
    "\n",
    "        # train actor\n",
    "        state_tensor = tf.convert_to_tensor(states, dtype=tf.float32)\n",
    "        with tf.GradientTape() as tape:\n",
    "            pi_tensor = self.actor_net(state_tensor)[0, action]\n",
    "            logpi_tensor = tf.math.log(tf.clip_by_value(pi_tensor,\n",
    "                    1e-6, 1.))\n",
    "            loss_tensor = -self.discount * td_error * logpi_tensor\n",
    "        grad_tensors = tape.gradient(loss_tensor, self.actor_net.variables)\n",
    "        self.actor_net.optimizer.apply_gradients(zip(\n",
    "                grad_tensors, self.actor_net.variables))\n",
    "\n",
    "        # train critic\n",
    "        self.critic_net.fit(states, np.array([[target,],]), verbose=0)\n",
    "\n",
    "\n",
    "agent = AdvantageActorCriticAgent(env)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "15:40:01 [INFO] ==== train ====\n",
      "15:43:01 [DEBUG] train episode 0: reward = -419.00, steps = 420\n",
      "15:46:29 [DEBUG] train episode 1: reward = -500.00, steps = 500\n",
      "15:49:50 [DEBUG] train episode 2: reward = -500.00, steps = 500\n",
      "15:53:12 [DEBUG] train episode 3: reward = -500.00, steps = 500\n",
      "15:56:33 [DEBUG] train episode 4: reward = -500.00, steps = 500\n",
      "15:59:55 [DEBUG] train episode 5: reward = -500.00, steps = 500\n",
      "16:03:19 [DEBUG] train episode 6: reward = -500.00, steps = 500\n",
      "16:06:40 [DEBUG] train episode 7: reward = -500.00, steps = 500\n",
      "16:09:58 [DEBUG] train episode 8: reward = -500.00, steps = 500\n",
      "16:13:06 [DEBUG] train episode 9: reward = -465.00, steps = 466\n",
      "16:14:25 [DEBUG] train episode 10: reward = -196.00, steps = 197\n",
      "16:15:41 [DEBUG] train episode 11: reward = -187.00, steps = 188\n",
      "16:16:57 [DEBUG] train episode 12: reward = -180.00, steps = 181\n",
      "16:18:31 [DEBUG] train episode 13: reward = -232.00, steps = 233\n",
      "16:19:40 [DEBUG] train episode 14: reward = -171.00, steps = 172\n",
      "16:20:35 [DEBUG] train episode 15: reward = -135.00, steps = 136\n",
      "16:21:33 [DEBUG] train episode 16: reward = -143.00, steps = 144\n",
      "16:22:42 [DEBUG] train episode 17: reward = -169.00, steps = 170\n",
      "16:23:32 [DEBUG] train episode 18: reward = -126.00, steps = 127\n",
      "16:25:01 [DEBUG] train episode 19: reward = -220.00, steps = 221\n",
      "16:26:04 [DEBUG] train episode 20: reward = -148.00, steps = 149\n",
      "16:27:35 [DEBUG] train episode 21: reward = -195.00, steps = 196\n",
      "16:28:35 [DEBUG] train episode 22: reward = -134.00, steps = 135\n",
      "16:29:40 [DEBUG] train episode 23: reward = -141.00, steps = 142\n",
      "16:31:33 [DEBUG] train episode 24: reward = -265.00, steps = 266\n",
      "16:32:40 [DEBUG] train episode 25: reward = -153.00, steps = 154\n",
      "16:33:43 [DEBUG] train episode 26: reward = -147.00, steps = 148\n",
      "16:34:39 [DEBUG] train episode 27: reward = -138.00, steps = 139\n",
      "16:35:39 [DEBUG] train episode 28: reward = -136.00, steps = 137\n",
      "16:36:39 [DEBUG] train episode 29: reward = -139.00, steps = 140\n",
      "16:37:38 [DEBUG] train episode 30: reward = -134.00, steps = 135\n",
      "16:39:33 [DEBUG] train episode 31: reward = -277.00, steps = 278\n",
      "16:40:25 [DEBUG] train episode 32: reward = -121.00, steps = 122\n",
      "16:41:22 [DEBUG] train episode 33: reward = -139.00, steps = 140\n",
      "16:42:11 [DEBUG] train episode 34: reward = -119.00, steps = 120\n",
      "16:43:08 [DEBUG] train episode 35: reward = -138.00, steps = 139\n",
      "16:44:06 [DEBUG] train episode 36: reward = -146.00, steps = 147\n",
      "16:44:51 [DEBUG] train episode 37: reward = -109.00, steps = 110\n",
      "16:45:45 [DEBUG] train episode 38: reward = -133.00, steps = 134\n",
      "16:46:32 [DEBUG] train episode 39: reward = -114.00, steps = 115\n",
      "16:47:16 [DEBUG] train episode 40: reward = -109.00, steps = 110\n",
      "16:48:03 [DEBUG] train episode 41: reward = -108.00, steps = 109\n",
      "16:48:46 [DEBUG] train episode 42: reward = -103.00, steps = 104\n",
      "16:49:35 [DEBUG] train episode 43: reward = -121.00, steps = 122\n",
      "16:50:15 [DEBUG] train episode 44: reward = -97.00, steps = 98\n",
      "16:50:15 [INFO] ==== test ====\n",
      "16:50:25 [DEBUG] test episode 0: reward = -110.00, steps = 111\n",
      "16:50:37 [DEBUG] test episode 1: reward = -123.00, steps = 124\n",
      "16:50:47 [DEBUG] test episode 2: reward = -104.00, steps = 105\n",
      "16:50:57 [DEBUG] test episode 3: reward = -95.00, steps = 96\n",
      "16:51:11 [DEBUG] test episode 4: reward = -136.00, steps = 137\n",
      "16:51:25 [DEBUG] test episode 5: reward = -143.00, steps = 144\n",
      "16:51:36 [DEBUG] test episode 6: reward = -114.00, steps = 115\n",
      "16:51:49 [DEBUG] test episode 7: reward = -135.00, steps = 136\n",
      "16:52:00 [DEBUG] test episode 8: reward = -122.00, steps = 123\n",
      "16:52:10 [DEBUG] test episode 9: reward = -104.00, steps = 105\n",
      "16:52:21 [DEBUG] test episode 10: reward = -112.00, steps = 113\n",
      "16:52:33 [DEBUG] test episode 11: reward = -132.00, steps = 133\n",
      "16:52:43 [DEBUG] test episode 12: reward = -97.00, steps = 98\n",
      "16:52:52 [DEBUG] test episode 13: reward = -98.00, steps = 99\n",
      "16:53:03 [DEBUG] test episode 14: reward = -121.00, steps = 122\n",
      "16:53:17 [DEBUG] test episode 15: reward = -163.00, steps = 164\n",
      "16:53:31 [DEBUG] test episode 16: reward = -152.00, steps = 153\n",
      "16:53:44 [DEBUG] test episode 17: reward = -132.00, steps = 133\n",
      "16:53:55 [DEBUG] test episode 18: reward = -115.00, steps = 116\n",
      "16:54:06 [DEBUG] test episode 19: reward = -120.00, steps = 121\n",
      "16:54:17 [DEBUG] test episode 20: reward = -125.00, steps = 126\n",
      "16:54:31 [DEBUG] test episode 21: reward = -154.00, steps = 155\n",
      "16:54:51 [DEBUG] test episode 22: reward = -184.00, steps = 185\n",
      "16:55:01 [DEBUG] test episode 23: reward = -99.00, steps = 100\n",
      "16:55:10 [DEBUG] test episode 24: reward = -92.00, steps = 93\n",
      "16:55:26 [DEBUG] test episode 25: reward = -174.00, steps = 175\n",
      "16:55:35 [DEBUG] test episode 26: reward = -101.00, steps = 102\n",
      "16:55:45 [DEBUG] test episode 27: reward = -99.00, steps = 100\n",
      "16:55:55 [DEBUG] test episode 28: reward = -114.00, steps = 115\n",
      "16:56:04 [DEBUG] test episode 29: reward = -86.00, steps = 87\n",
      "16:56:12 [DEBUG] test episode 30: reward = -90.00, steps = 91\n",
      "16:56:23 [DEBUG] test episode 31: reward = -112.00, steps = 113\n",
      "16:56:32 [DEBUG] test episode 32: reward = -98.00, steps = 99\n",
      "16:56:42 [DEBUG] test episode 33: reward = -104.00, steps = 105\n",
      "16:56:54 [DEBUG] test episode 34: reward = -131.00, steps = 132\n",
      "16:57:11 [DEBUG] test episode 35: reward = -168.00, steps = 169\n",
      "16:57:25 [DEBUG] test episode 36: reward = -136.00, steps = 137\n",
      "16:57:34 [DEBUG] test episode 37: reward = -97.00, steps = 98\n",
      "16:57:50 [DEBUG] test episode 38: reward = -178.00, steps = 179\n",
      "16:57:57 [DEBUG] test episode 39: reward = -80.00, steps = 81\n",
      "16:58:15 [DEBUG] test episode 40: reward = -194.00, steps = 195\n",
      "16:58:26 [DEBUG] test episode 41: reward = -119.00, steps = 120\n",
      "16:58:36 [DEBUG] test episode 42: reward = -119.00, steps = 120\n",
      "16:58:44 [DEBUG] test episode 43: reward = -91.00, steps = 92\n",
      "16:58:56 [DEBUG] test episode 44: reward = -125.00, steps = 126\n",
      "16:59:09 [DEBUG] test episode 45: reward = -145.00, steps = 146\n",
      "16:59:18 [DEBUG] test episode 46: reward = -98.00, steps = 99\n",
      "16:59:27 [DEBUG] test episode 47: reward = -105.00, steps = 106\n",
      "16:59:37 [DEBUG] test episode 48: reward = -103.00, steps = 104\n",
      "16:59:55 [DEBUG] test episode 49: reward = -204.00, steps = 205\n",
      "17:00:04 [DEBUG] test episode 50: reward = -98.00, steps = 99\n",
      "17:00:13 [DEBUG] test episode 51: reward = -107.00, steps = 108\n",
      "17:00:24 [DEBUG] test episode 52: reward = -117.00, steps = 118\n",
      "17:00:36 [DEBUG] test episode 53: reward = -136.00, steps = 137\n",
      "17:00:45 [DEBUG] test episode 54: reward = -104.00, steps = 105\n",
      "17:00:53 [DEBUG] test episode 55: reward = -93.00, steps = 94\n",
      "17:01:06 [DEBUG] test episode 56: reward = -135.00, steps = 136\n",
      "17:01:16 [DEBUG] test episode 57: reward = -113.00, steps = 114\n",
      "17:01:26 [DEBUG] test episode 58: reward = -120.00, steps = 121\n",
      "17:01:35 [DEBUG] test episode 59: reward = -106.00, steps = 107\n",
      "17:01:46 [DEBUG] test episode 60: reward = -117.00, steps = 118\n",
      "17:01:54 [DEBUG] test episode 61: reward = -93.00, steps = 94\n",
      "17:02:07 [DEBUG] test episode 62: reward = -149.00, steps = 150\n",
      "17:02:17 [DEBUG] test episode 63: reward = -117.00, steps = 118\n",
      "17:02:30 [DEBUG] test episode 64: reward = -144.00, steps = 145\n",
      "17:02:41 [DEBUG] test episode 65: reward = -129.00, steps = 130\n",
      "17:02:53 [DEBUG] test episode 66: reward = -134.00, steps = 135\n",
      "17:03:09 [DEBUG] test episode 67: reward = -175.00, steps = 176\n",
      "17:03:18 [DEBUG] test episode 68: reward = -108.00, steps = 109\n",
      "17:03:29 [DEBUG] test episode 69: reward = -118.00, steps = 119\n",
      "17:03:37 [DEBUG] test episode 70: reward = -98.00, steps = 99\n",
      "17:03:46 [DEBUG] test episode 71: reward = -101.00, steps = 102\n",
      "17:04:07 [DEBUG] test episode 72: reward = -228.00, steps = 229\n",
      "17:04:16 [DEBUG] test episode 73: reward = -107.00, steps = 108\n",
      "17:04:32 [DEBUG] test episode 74: reward = -173.00, steps = 174\n",
      "17:04:42 [DEBUG] test episode 75: reward = -106.00, steps = 107\n",
      "17:04:56 [DEBUG] test episode 76: reward = -143.00, steps = 144\n",
      "17:05:09 [DEBUG] test episode 77: reward = -131.00, steps = 132\n",
      "17:05:21 [DEBUG] test episode 78: reward = -111.00, steps = 112\n",
      "17:05:33 [DEBUG] test episode 79: reward = -127.00, steps = 128\n",
      "17:05:42 [DEBUG] test episode 80: reward = -103.00, steps = 104\n",
      "17:05:53 [DEBUG] test episode 81: reward = -112.00, steps = 113\n",
      "17:06:07 [DEBUG] test episode 82: reward = -150.00, steps = 151\n",
      "17:06:16 [DEBUG] test episode 83: reward = -101.00, steps = 102\n",
      "17:06:28 [DEBUG] test episode 84: reward = -132.00, steps = 133\n",
      "17:06:38 [DEBUG] test episode 85: reward = -99.00, steps = 100\n",
      "17:06:50 [DEBUG] test episode 86: reward = -111.00, steps = 112\n",
      "17:06:59 [DEBUG] test episode 87: reward = -92.00, steps = 93\n",
      "17:07:19 [DEBUG] test episode 88: reward = -210.00, steps = 211\n",
      "17:07:32 [DEBUG] test episode 89: reward = -131.00, steps = 132\n",
      "17:07:41 [DEBUG] test episode 90: reward = -99.00, steps = 100\n",
      "17:07:51 [DEBUG] test episode 91: reward = -115.00, steps = 116\n",
      "17:08:02 [DEBUG] test episode 92: reward = -102.00, steps = 103\n",
      "17:08:13 [DEBUG] test episode 93: reward = -101.00, steps = 102\n",
      "17:08:25 [DEBUG] test episode 94: reward = -108.00, steps = 109\n",
      "17:08:38 [DEBUG] test episode 95: reward = -137.00, steps = 138\n",
      "17:08:50 [DEBUG] test episode 96: reward = -134.00, steps = 135\n",
      "17:09:02 [DEBUG] test episode 97: reward = -128.00, steps = 129\n",
      "17:09:18 [DEBUG] test episode 98: reward = -188.00, steps = 189\n",
      "17:09:25 [DEBUG] test episode 99: reward = -83.00, steps = 84\n",
      "17:09:25 [INFO] average episode reward = -123.27 ± 29.22\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX8AAAD4CAYAAAAEhuazAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deXib5Zno/+8t77uTeLcTnMXZgUBMmgJhaWgJpQXaAk2ZHrqcMxkYmNJpOz1QetqZ9uKcmdNtYFrgStsZyu9AIW0GQtm3sg0UcCAkzr7Ha2In3iTbkiU9vz/0ypFtSZYj27L83p/r0oX8vFoeC7j96FnuW4wxKKWUshdHojuglFJq8mnwV0opG9Lgr5RSNqTBXymlbEiDv1JK2VBqojsQq6KiIlNdXZ3obiilVFLZunVruzGmeHh70gT/6upq6urqEt0NpZRKKiJyNFy7TvsopZQNafBXSikb0uCvlFI2pMFfKaVsSIO/UkrZkAZ/pZSyIQ3+SillQ3EFfxG5QUR2iohfRGqHXbtLRA6IyF4RuTKkfaWI7LCu3SciEk8flFJqutrb2sMvXtpHn8c37q8d78i/Hvg88EZoo4gsBdYDy4B1wP0ikmJdfgDYANRYt3Vx9kEppaYdYwz/9KedPPT2EfoHpljwN8bsNsbsDXPpWuAxY4zbGHMYOACsEpFyIN8Y844JVJF5GLgunj4opdR09Fx9K28fPMl3PrWQGTnp4/76EzXnXwk0hPzcaLVVWveHt4clIhtEpE5E6tra2iako0opNZG8Pj+v72vj317ZT6/HG9Nz+jw+7nlmN0vK87npY2dNSL9Gze0jIi8DZWEu3W2M2RLpaWHaTJT2sIwxG4GNALW1tVpvUimVFIwxbGvoZMu2Zp7e3ky70wPA/hNO7l2/gtGWOh94/SBNnX38/MZzSXFMzLLoqMHfGHPFGbxuIzA75OcqoNlqrwrTrpRSScsYQ9+Aj8aOPp7e3sJT25o4crKX9FQHaxeXcO2KSna3dHPvK/s5b04hX7tobsTXajjVy4OvH+Sacyv42LxZE9bnicrq+RTwqIj8HKggsLD7njHGJyI9IrIaeBe4Gfi3CeqDUsrmHn33GPOKc1g9DkHU6/PzXwdP8tKuVo53u+nqHaCzz0Nn7wCdfQN4vH4ARODj82bxt5ct4MrlZRRkpQHwqaWl7Gzu4p5ndrO8soALqmeGfZ97ntlNigh3fXpx3H2OJq7gLyKfIxC8i4FnRGSbMeZKY8xOEdkE7AK8wG3GmOBy9a3AQ0AW8Jx1U0qpcXW8u5/vP7mDotwMXv3OZeRmjD3cGWP4qLGLJz9s4untLbQ73eRmpFI1I4vC7DTmFeVSmJ1GYXY6hdlpzMxJ55KaYsoKMke8lsMh/OzGFVz7y7f420c+4Jm/u5iS/KGPe2t/O8/vbOUfrlxEeUHWGf/usZDAppupr7a21mg+f6VUrB547SD/8vweAP7m0nncddWSmJ979KSLzR80hZ2+uWxRMZlpKaO/SAR7Wrv53K/eZnllPo/+9WrSUgL7bgZ8fq66900GfH5e+OYlcb1HKBHZaoypHd6eNMVclFIqVsYY/lDXwKrqmcyZlc2/v3WYL9bOZl5x7qjP/aihkxsefIcBv58L54+cvonX4rJ8/vkLZ3PHY9v438/u5oefXQbA794+woETTn5zc+24Bf5oNPgrpaadrUc7ONTu4tbL5nPZohJeqG/lR0/v4j++ekHUnTbd/QPc/vsPKM7LYNMtH6eycGKmXq5dUcm2hk7+47+OsGJ2IRfOL+Lel/dz2aJi1i4pmZD3HE5z+yilpoTndrTwy1f3Mx5T0X+oayQ7PYVPn11OcV4Gd1xRw2t723h1z4mIzzHGcOfm7TR39nPfl86bsMAf9L1PL+GC6hncuXkH39q0jX6vjx98Zumo20DHiwZ/pdS4+qihk7uf2DGmlAR7Wru54/Ft/PTFfWyqaxj9CVH0erw8vb2Zz5xTTo61yPuVC6tZUJLLj57eFbFfj7x7jGd3BBZbV541I64+xCItxcGvbjqf3MxU3tzfztcvnhvTtNR40eCvlBpXz+9s5ZF3j/FPf9oZ0+P7B3x887Ft5GemsmruTH741E72tvac8fs/u6MVl8fHDbWnjxqlpTj44WeXcvRkL7996/CI5+xq7uZHT+/i0oXFbFgz74zfe6xK8jP59c21XL+yir/7RM2kvS9o8FfTjDEmrsCh4tfe4wbg9+818Pj7x0Z9/E9f2Mue1h5+cv25/PKm88jNSOP2Rz+IORXCcJvqGphblEPtsNH7mppirlxWyi9fPUBLV99gu8vt5fZHP2BGdho/v/FcHBN0ojaSFbML+ekN557RVtR4aPBX08qWbc1c+a9vsP+4/gFIlHanmyXl+aypKeJ/bdnJ9sbOiI/9rwPt/Oatw/y31Wdx+eISSvIy+dcvruBAm5N/fCq2bw6hjrS7eO/wKa5fWRV27vz7Vy/Fbwz/59nAFlBjDN9/sp4jJ13cu/48ZuVmjPk9k5UGfzWt/OmjQLaQ3Tr6T5h2p4fS/AzuXX8exbkZ3Pr/PuCUyzPicZ29Hr696SPmF+fwvU+f3oN/cU0Rt1++gE11jTzxYeOI50Xzx62NOAS+cH5V2OuzZ2bzN5fO56mPmnnv8Cn+uLWRJz5s4o61C8flFHAy0eCvpo3u/gHe3N8OwOE2V4J7Y1/tTjdFuRnMzEnngS+fT5vTzTd+/yE+/+ldPMYYvvfEDk663Ny7/jyy0ofua79jbQ2rqmdy9xP1HGpzxvS+Pr9h8weNXLIw/AnboFsvnU9lYRZ3bt7OD7bs5OPzZnH7Jxac2S+bxDT4q2nj1d0n8Pj8pDqEw+2xBYx4nejujzqtkczaety8fbB9TM8xxgwGf4Bzqgr58bXLeOtAOz978XTpj80fNPHsjla+9clFLK8sGPE6qSkO7v3SCjJSHdz26Icx7Rx660A7LV393Fg7O+rjstJTuPvqJRxqd5GdnsK961dMWObMqUwPeamE6fP42PD/1XFuVSG3XjZ/cFvemXp2Rwtl+ZnML8nhUPvkjPy//2Q9L+46zqq5M/nm2ho+Pn/WpO3THk0gEHs4etJFc1c/Z83MZkl5Pumpkcd8TreXF+pbeXJbE/91oB2/gee/uYbFZfkxvWdX3wADPkNR7uniI1+8YA7bGjq5/7WDnDu7kCVl+fxwSz2r5s5kwyWRd9aUF2TxsxvP5esP1XHPM7v58XXLo773proGCrPTYjokddXyMu68ajGr580akV/HLjT4q4R5ZkcLb+5v58397Txe18A/XLmI68+vOqPdFk63l9f2tXHTqjn4jeGJD5owxkxoIPb7De8dOcXS8nyOnnRx02/e5YLqGdyxdiEXLZjcPwJ+v+H5na3sbO7iSHsvR066OHqyF6d76I6Z9BQHSyvyWTG7cPBWXpjJG/vaeXJbEy/vOo7b66dqRhbXrqjkiQ+baDzVF3Pwb3cGdvoU5w1dOP3Ha5axq7mbb2/6iOqibBwOiSlX/ScWl/LXa+by6zcPU1s9g2tXhK/91Nnr4aWdx7npY3PISB09NYKIcMul82P6naYrDf4qYR5//xjzinL46Y3n8uOnd/HdP27nd28f4X99ZumYF9/+vOcEHq+fT59dzq7mLnrcXtqcbkryJm5Ud6jdSWfvAN+7agnXrKjg8fcbeOC1g3z5t++y8qwZ3LG2hjU1RRP+R2Df8R7u3LydD451kuoQZs/M5qxZ2VxQPZPqWdmcVZRDeUEmh9pcbGvoZFtDJ4+/38BDbx8BIMUh+PyGGdlp3Fg7m+vOq+D8OTNo6erniQ+bOGFt3YxFW09gYbdo2K6ZjNQU7v/ySj77b29R39TNvetXUDUjO6bX/IcrF1N3tIM7HtvGcztauevTizlrVs6Qx2zZ1ozH5x91ykedpsFfJcSBEz28f6SDu65azPlzZvCft17In7a38C/P7WH9xr+wbllZ2P/JI3muvoXivAxWnjWDPmt++HCba0KDf92RDgBWVs8gMy2Fr1xYzRcvmM0f6hq4/7WD3Pzv7/H1i+byg88unZD3d3t9/OrVAzzw+kFyM1L5+Y3n8tlzKwazRA63uCyfT59dDgRy0+8/4WRbQyeH212snjeTNTXFQ54bHL2f6OmPuU/Bkf/w4A9QWZjF7762iu1NnRFH8OGkpzp49H+s5jdvHuL+1w7y6p4TfO2iam77xALyMwPJ1v6wtYFlFfksrYjtG4rS4K8S5PH3G0h1CJ+3tuSJCNecW8Gnlpby27cO86s/H+DVn5/goa9dwIULiqK+Vp/Hx5/3tHH9yipSHMK8osAfjEPtrgmthFR3tIOZOemD7weQmZbCf/t4NTdeMJuvP/Q+r+09MSHB//0jp7hz83YOtrn43HmVfP/qJWPao56a4mBJeT5LyiMHy7QUB7Ny0sc08o807RN0dlUBZ1eNXOAdTVZ6Cn+3toYbL5jNT17Yy8Y3D/HHrY38/ScXsmJ2IfVN3fzTNcvG/Lp2FtduHxG5QUR2iohfRGpD2qtFpE9Etlm3B0OurRSRHSJyQETuk6myOqYmjcfrZ/MHTVyxpHREkMhMS+G2yxfw2ncuo6wgkx89vWvIFsFwXt93gr4BH1edHSg1XVGYRXqqg8MTvOi79WgHK8+aEXZaJyM1hSVl+TR39Y1LorKg7v4BvvfEDm548B36B/z87uur+MUXV0zY4aTivAxOdI9t5J/iEArHKf3xcKX5mfz0hnP50+0XM78kl+8/Wc/1D75NeoqDa1dUTMh7TlfxbvWsBz4PvBHm2kFjzArrdktI+wPABgKlHWuAdXH2QSWZl3Yd55TLw/pVkednS/Iz+YcrF7GntYcnP2yK+nrP7mhlVk46q6yyeCkOoXpWNofGsNf/RHc/z9e3xPz4dqebw+2uESkEQpUXZtE/4KezdyDm142ktaufn724l8t/8hqPvXeM/37xXF78+0u4dGFx3K8dTUl+5thG/j0eZuWkT3iKhOWVBTy+YTUPfvl8KgqzuL62isLs9NGfqAbFNe1jjNkNxLygJSLlQL4x5h3r54eB69BSjrby2PvHqCjIZE1N9MB19dnlbHzjED9/aR9Xn1MetsBF/4CPV3Yf55oVlaSGzFfPK8pl34nYT/n++s1D/PrNw7z53cuZPXP0hcjgfH9tdeTgX2EdNGru6mNGztgDkzGGuqMdPPT2EV6ob8VnDGsXl/CNtTWcU1U45tc7E6V5Gewbw2nptpA9/hNNRFi3vJx1y8sn5f2mm4k85DVXRD4UkddFZI3VVgmEntdutNrCEpENIlInInVtbW0T2FU1WRpO9fLWgXZuqJ096jY/h0O466rFNHX28fA7R8I+5s397bg8Pq5aXjakfW5xDsdO9uL1+WPq146mLoCo+d5DbT16ivRUR9gDSkHlVj74ls7Yp00g8Adt0/sNXH3fW9zw4Du8ua+Nr11UzevfuZzffOWCSQv8ACX5GbQ53fhHmXoLane6KYow36+mllFH/iLyMlAW5tLdxpgtEZ7WAswxxpwUkZXAkyKyDAj3f3vE/6qMMRuBjRCo4TtaX9XU94etgb/9N9SGz70y3IULirh0YTG/+vNBvlg7h4LsoXPJz+1ooSArjY/PH7qwO7coB6/f0NDRx9yi6DuG/H7DzqZuAF7efZyvXFg9ar/qjnZwTmVB1D3loSP/sbj5t+/x3pFTLCrN439/7myuO6+C7PTE7M0oycvE5zecdHkiLuKGau9xs6Bk8nLSqzM36sjfGHOFMWZ5mFukwI8xxm2MOWnd3wocBBYSGOmH/l9fBTTH9yuoZOHzB+qqXlJTHPMeb4A7r1pMd/8A9792YEi72+vjpd3H+dTS0hHbG+cXBwJ+LGkeGjp66XF7KcrN4N1Dp0YcjBquf8BHfVMXtdYaQyRFuRmkpQjNYxj5D/j81B09xVcvrOb5b67hpo/NSVjgBygZw3bP4IniWP5IqMSbkGkfESkWkRTr/jwCC7uHjDEtQI+IrLZ2+dwMRPwjoqaXN/a10dLVz/oLxnYQZ0l5Pp87r5L/ePsIzZ2nR9FvHzhJT793cO96qLlFgdFnLIu+9dao/5ZL5+Hx+Xlrf/Qpxu2NXQz4TNTFXghMW5XmZw7JHT+a1q5+/AaWludPiTQRwdQHsSz6dvd78fj8FNsoLXIyi3er5+dEpBH4OPCMiLxgXboE2C4iHwF/BG4xxpyyrt0K/AY4QOAbgS722sRj7x9jVk46a5eUjvm53/7UIgB+/tK+wbZnd7SQl5nKhQtG7uWfmZNOYXZaTDl+djR1kZYi3PSxOeRnpvLy7ujz/u8fCfynHEupv4qCrDHN+Tec6gWgasbE1o+NVXDk39Y9evCPdsBLTT3x7vZ5AngiTPtmYHOE59QB0TM0qWnnRE8/r+w+wX+/eG7UxGKRVBZm8dULq/n1m4f4H2vmMr84lxd3HeeTS0ojzrvPLcqJKbXzzuYuFpbmkZ2eymWLSvjznhP4/CbigvTWox3ML86JaQdPRWEmdUc7Rn1cUGNH4FtCLDuOJkNwCud4DHv9gxW8NPgnB03prCbF5q1NeP2GG8c45RPqby+bT15GKv/y3B7eOXiSrr4Brgoz5RM0ryiXQ6PM+RtjqG/qYnlFYNfO2iUlnHR52NYQPk2z32/YerSD2rOiz/cHlRdmcby7P+bdMo0dvTiEqPnoJ1NmWgoFWWkxTfu0BUf+ebrfPhlo8FcTzhjD4+8fY1X1TOYXn/lOkMLsdP728gX8eW8b//L8HnLSU1hTEzn1w7ziHI53u3FFWcBt7uqno3eA5ZWBNAeXLSwhxSG8svt42McfbHPS1TcQdX9/qIqCTAZ8ZnBKZDSNHX2UF2RFzM+TCKX5GTEt+OrIP7lMnf/C1LT1l0OnOHKyN+qJ3lh99cJqygsy2dnczdolpWEPfgUFt3hGS/NQb+3vX2bt1y/ITqP2rBkR9/sHp3BG2+kTVF4QmLtv7opt3r+ho5fKKTLfH1SSl8nxmOb8PTgEZuhJ26SgwV9NuMffP0ZeZipXjcNJzMy0lMHF36vPif5684pPJ3iLZGdTFykOYWlIgrMrlpSyp7WHxo7eEY9//8gpZuWkUz0rtjn58kJrr39nbDt+Gjv6mD2GbbCToSQvg7YYpn3anW5m5WbYsipWMtLgryZU/4CP5+pbuebcihF1Ws/UF86v5MnbLuJTS6PvGqqelYNI9Hq+9c3dLCjOHfINIlgJ6pUwu36iJXMLpyI48o8h+Hu8flq7+6fMTp+g4vxA8B8tQV37JKZ2UPHT4K8m1I6mLtxe/7gmIBMRVswuHDUAZ6alUFGQFXXRd0dTF8sqh6Y1nlecy7yiHF4eNu/f1uPm6MnemOf7AQqz08hMc9ASw7RPS1cfxkydbZ5BpXmZeHyjJ6hrc3qGlG9UU5sGfzWhBguexLAnfiLMK86JOOd/orufth734E6fUGuXlIw47bv1aHB/f2zz/RD4Q1VRkBXTQa/gNs+xnH6eDCX51nbPURZ923vcesAriWjwVxNq69FTzCvKmbB886MJ7vUPN2VR3xxY7A2XnG3tktIRp33rjnSQkeoY3BkUq4rCrJhSPAQPeM2eObVG/sFqaCeiLPoaYwIZPTW1Q9LQ4K9isrulm289vo1N7zfE/JzgnvhEjfoB5hXlDNbzHa6+qRsRwpb+W3nWjBGnfeuOdnBuVWFMBcJDlRfEluKhsaOPFIdQlj819vgHnc7vEzn497i9eLx+nfZJIlrGUUW1s7mL+17Zzws7A/Pf2xo6Yz6odajdSUfvABfEuC1yIsy1zhWEq+db39TF3KIccjNG/m+QluIYctrX4/VT39TFX18yb8x9KC/M4kSPmwGfP+r+/caOXsoLMofUJZgKgtM+0fb6B/f4a1K35KHBX4VV39TFva/s56Vdx8nLTOUba2sQ4N5X9tPY0RvTvHRogfNEiVbPd2dzd9RvJWuXlPDUR81sa+hkwOfH6x89mVs4FQWZGBNIkRDtc2vs6Jtyi70A2emp5GWkRp32aXd6AD3glUw0+KshdjV38/OX9vLy7hPkZ6byzStq+NpFcynISmP/8R7ufWU/b+1vZ/2qOaO+VrgC55MtUj3fUy4PTZ19fOXCsyI+N/S0b4717eBMprCCRV2aO6MH/4aO3lGrmyVK8SinfDWpW/LR4K8A6PP4+MXL+/jtW4fJzUjlW59cyFcvqiY/83TxlAUluZTmZ/BmjMF/69EOzp8T+574iRCpnu/O4GJvmJ0+QQXZaVxQHTjtW16QSU1J7hnViQ0WdYk27+/2+jje7Z5yB7yCSvIyRhn5a/BPNhr8FW/tb+d7T+zg2KlevrRqNneuWzKiYhYEti2uqSnm5d3Ho2a9hNMFzr8YRyK38RKunm+wbOOyKMEfYO3iUu55djdHTrr43HkRK45GFTryjyR4bSpO+wCU5mfy4bHwye4gMOfvkEAqbZUcptbKkppUHS4P3970EV/+7bukOoTHNqzm/3z+nLCBP2hNTRGdvQODI+dItlo5cC5I4Hx/ULh6vjubupk9Myvq7wqnT/v2D/jHtL8/VG5GKvmZqVFH/sFUElM1+JfkZXC8uz/iKd82p5uZOema2iGJ6MjfhowxPPVRMz/60y66+ga4/fIF3P6JBVGTpAVdtCCQRfPN/e1RC4nXHRm9wPlkmRemnm99c1fUKZ/B51qnfQ+1u85osTdotL3+DaesA15TJI//cCV5mbi9frr7vRRkjfyD2dbj0SmfJBNvJa+fiMgeEdkuIk+ISGHItbtE5ICI7BWRK0PaV4rIDuvafTIVatXZzP2vHeSOx7Yxe2Y2T3/jYr5z5aKYAj8E5nSXlufz5iilDmMpcD5Z5g2r59vVN8DRk70x/2H6/PmVLCzN5awYk7mFM9pe/8aOXlKn4B7/oOB2z7YIi77tTrdu80wy8U77vAQsN8acA+wD7gIQkaXAemAZsA64P1jTF3gA2ECgrm+NdV1Nog+OdrCgJJfNt17I4rKxnVYFWLOwiK1HOyLmyQ8WOE/kFs9Q84bV893VHKjZG2vwv+3yBbz495fGtXBdXpgVNb9PY0cfFYVZU3baZLRTvprULfnEFfyNMS8aY4IR4C9AlXX/WuAxY4zbGHOYQL3eVSJSDuQbY94xgcnDh4Hr4umDGjun2xvX/OyaBcUM+AzvHT4V9vrpAueJO9wVasawer7B9YplYU72hjMeX04rCjI55fLQP+ALez1wdmJqzvdD9Pw+xhgr+OtibzIZzwXfr3O6GHslEJoHoNFqq7TuD28PS0Q2iEidiNS1tUWfZlCxc3m85IU51Rqr2uoZZKQ6eCPC1E/d0dgLnE+W0Hq+9U1dlBdkTupINVjUJdLov2GKHvAKGkzxEGbk7/L46B/w68g/yYwa/EXkZRGpD3O7NuQxdwNe4JFgU5iXMlHawzLGbDTG1BpjaouLp+bhl2Tk7PcOHlo6E5lpKayaO5O39reHvV53JFDgfCpt+wut51vf3D3qFs/xFq2oS/+Aj7aeqbvHHwI7lrLSUsLm99Hyjclp1OBvjLnCGLM8zG0LgIh8BfgM8Ffm9D6wRiB0g3cV0Gy1V4VpV5PI6faRmxnfRq81NUXsP+EcsYg51gLnkyVYz/dETz8H25xjzswZr2hFXZo6gzt9pu7IX0SsWr4jg//pwu0a/JNJvLt91gH/E7jGGBNa8+4pYL2IZIjIXAILu+8ZY1qAHhFZbe3yuRnYEk8f1Ng53QNhk5mNRTANwfDRf7DA+VRZ7A0Kpph4bkcrxsDZk7wFtWzwlO/IaZ+pmsd/uEAt35H9H0zqpiP/pBLvnP8vgTzgJRHZJiIPAhhjdgKbgF3A88BtxpjgStetwG8ILAIf5PQ6gZoEXp+f/gE/OenxBf/FZXkU5Wbw1oGhwX+wwPkUmu+HwEEvgKc+CnzRnOzzB5lpKRTlpofd7hnM4z+V5/zhdDnH4QZTO+RNnWk+Nbq4IoAxZkGUa/cA94RprwOWx/O+6sy53IG/wfFO+wRSPRTxxr42/H6Dw9o5VHekg1k56YOHqaaKYD3frUc7KMrNGFzAnEzlBeEPejV29JGWIiNSTk81JXkZvBZm5N/m9CACM88g75FKHE3vYDNOT2Bnbm5G/IevLl5QxEmXh92t3YNtdUdPjanA+WQJ1vMFWF6Zn5D+RTro1djRS+UU3uMfVJqficvjG1LaEgIj/5nZ6VOuDoGKTv9t2UzwYFZuRvScNrFYU3M61QOcWYHzyRQ86RtLWoeJUFGYRUuEkf9Un++H0O2eQ3+H9h494JWMNPjbTE9/IPjnjMPIvyQ/k0WleYOLvmdS4HwyBRd9J3unT1B5QSY9bi/d/QND2qf6Aa+gwVO+w+b9251une9PQhr8bSY48s+Lc84/aE1NEe8dOUX/gI+6Ix1WMrfEBNfRLCrLxyFwdpSEdBMpmNo5dPTf5/HR7vQkR/DPD1/Lt01TOyQlDf42E5yvjeeQV6iLa4rweP28d/iUVeB8aiRzC+f6lVU8dfvFVBYmJtAGi7o0h8z7N3UGdvrMnqLZPEOVDub3GT7t49FtnklIg7/NDAb/OLd6Bn1s7izSUxy8tOs49U1d1CawWPtoEp1iuiLMyL9hcI//1B/552elkp7qGDLyd7m99A349IBXEtLgbzPO/vGd9slKT6G2egab6hrOuMC5XZTkZeCQoeUcGwf3+E/9kb+IWOUcT//x0vKNyUuDv824xnnaBwJTP25voErWVErmNtWkpjgozc8cste/saOP9FRH0kyblOQNTfFwOvjrgm+y0eBvM06Pl4xUB2njuCf7EivVw4IzLHBuJ8P3+jd29FFVmDV4SG6qK83PHBL823o8gI78k5EGf5tx9nvjzusz3NLyfCoKMrnYKvGoIhte1KWxo5fKJJjvDwrW8g0Kjvy1ilfy0Rq+NuNye+NO7TCcwyE88401ZKVPzV0+U0lFQSYv7zqOMQYRobGjj08l6NDZmSjJz6Sn30v/gI/MtBTaetyB1A5TKH23io2O/G3G6faO206fUDNy0mOuA2xn5QVZuL1+Trk8uNxeTrqSY49/UPGwoi7tTjczstPHdRpRTQ79N2YzTvf4T/uo2A1u9+zqH8zjnwx7/Amu8LIAABRsSURBVINK84OnfANTP1q+MXlpFLAZl9un87MJVBFS0Ss1JbDIm0wj/2B+n+ODI3+PLvYmKR3524zTHV8JRxWf0Fq+jUl0wCtoMLnbkJG/Bv9kFG8lr5+IyB4R2S4iT4hIodVeLSJ9VoGXwSIv1rWVIrJDRA6IyH0y1XL/TnM67ZNYs3LSSU9x0NzVR8OpXjKSaI8/wIzsdFIdMrjdUzN6Jq94R/4vAcuNMecA+4C7Qq4dNMassG63hLQ/AGwgUNqxBlgXZx/UGAS2eurCbKI4HEJZQSYtnYGRf+WMrClX+yAah0MGt3v2ery4PD7N6Jmk4gr+xpgXjTHByg5/YWhx9hFEpBzIN8a8YxV7fxi4Lp4+qNj5/Ia+AZ9O+yRY8KBXY0cfs5MgrcNwxfmZtPW4abcOeCXTNxd12njO+X+dofV454rIhyLyuoissdoqgcaQxzRabWGJyAYRqRORura2tnHsqj05Bwu5aPBPpIrCQDnHZMnjP1wgv4+btsHavRr8k9GoUUBEXgbKwly62xizxXrM3YAXeMS61gLMMcacFJGVwJMisgwI9/3WRHpvY8xGYCNAbW1txMep2Lg0+E8JwZG/3yRHQrfhSvIyqDty6vTpXh35J6VRo4Ax5opo10XkK8BngLXWVA7GGDfgtu5vFZGDwEICI/3QqaEqoPnMuq7GajD4j/MJXzU2FYVZ+K2hTDKO/EvzM+noHaDZOqegC77JKd7dPuuA/wlcY4zpDWkvFpEU6/48Agu7h4wxLUCPiKy2dvncDGyJpw8qdj0TkNFTjV1wrz8k1wGvoOB2zz0tPQDM0kNeSSneKPBLIAN4ydqx8BdrZ88lwI9ExAv4gFuMMaes59wKPARkEVgjeG74i6qJMVjCUYN/QgX3+kNyjvyD5Rx3tXRTmJ2mqR2SVFxRwBizIEL7ZmBzhGt1wPJ43ledGWe/jvynggor+GemOZiVhAnRgoXc97b2MGdW8n1zUQEaBWxEd/tMDflZqWSnp1BRmFx7/IOCI3+Pz6+LvUlMo4CN6G6fqUFEmDMzOyl3+gDMygmUo/Qb3eaZzDQK2IhTF3ynjH/70nlJW/8gxSEU5QbKOWpGz+SlUcBGnG4f6akO0lN1gS7RakrzEt2FuJTkB4O/jvyTlUYBG3G6B3TKR42LUmvRV+f8k5cGfxtxuX0a/NW4CC76alK35KXB30Z6+jWXvxofxYMj/8xRHqmmKg3+NuJyazpnNT7mFmWT6hDKCzX4JysdBtqIy+NNykNFaur57DkVnFNVqAu+SUxH/jbi1GkfNU5SUxzML85NdDdUHDT424jT7SVPM3oqpdDgbytOt5ecdA3+SikN/rbh8xt6PT7N5a+UAjT424bLo3l9lFKnafC3CZfm9VFKhdDgbxOa0VMpFSreMo4/FpHtIrJNRF4UkYqQa3eJyAER2SsiV4a0rxSRHda1+yQZE5onoZ5+Df5KqdPiHfn/xBhzjjFmBfA08AMAEVkKrAeWAeuA+4M1fYEHgA0E6vrWWNfVBHO5fYAWb1dKBcQV/I0x3SE/5gDGun8t8Jgxxm2MOQwcAFaJSDmQb4x5xxhjgIeB6+Lpg4qN0z0AoFs9lVLAOKR3EJF7gJuBLuByq7kS+EvIwxqttgHr/vD2SK+9gcC3BObMmRNvV23NGRz567SPUooYRv4i8rKI1Ie5XQtgjLnbGDMbeAS4Pfi0MC9lorSHZYzZaIypNcbUFhcXj/7bqIgGF3x12kcpRQwjf2PMFTG+1qPAM8APCYzoZ4dcqwKarfaqMO1qgp0u4ahZPZVS8e/2qQn58Rpgj3X/KWC9iGSIyFwCC7vvGWNagB4RWW3t8rkZ2BJPH1RsnG4v6SkOMlI1+Cul4p/z/2cRWQT4gaPALQDGmJ0isgnYBXiB24wxPus5twIPAVnAc9ZNTbBARk8N/EqpgLiCvzHmC1Gu3QPcE6a9Dlgez/uqsXO5vTrfr5QapCd8baJHM3oqpUJo8LeJQAlHDf5KqQAN/jah0z5KqVAa/G2ix60lHJVSp2nwtwmX20ueBn+llEWDv01o8XalVCgN/jbg9xtcHp8u+CqlBmnwtwEt4aiUGk6Dvw0Ec/nrtI9SKkiDvw04NaOnUmoYDf42MBj8NbePUsqiwd8GThdvT0twT5RSU4UGfxsIFm/XrJ5KqSAN/jYQHPnn6chfKWXR4G8Dwa2eOvJXSgXFW8nrxyKyXUS2iciLIlJhtVeLSJ/Vvk1EHgx5zkoR2SEiB0TkPquil5pAp6d9dLePUiog3pH/T4wx5xhjVgBPAz8IuXbQGLPCut0S0v4AsIFAaccaYF2cfVCjcLm9pKUIGan6RU8pFRBXNDDGdIf8mAOYaI8XkXIg3xjzjjHGAA8D18XTBzU6p5XRU79kKaWC4h4Kisg9ItIA/BVDR/5zReRDEXldRNZYbZVAY8hjGq22SK+9QUTqRKSura0t3q7allMLuSilhhk1+IvIyyJSH+Z2LYAx5m5jzGzgEeB262ktwBxjzHnAt4BHRSQfCDf0jPhtwRiz0RhTa4ypLS4uHuvvpizOfg3+SqmhRo0IxpgrYnytR4FngB8aY9yA23r+VhE5CCwkMNKvCnlOFdA8ph6rMXN5NJ2zUmqoeHf71IT8eA2wx2ovFpEU6/48Agu7h4wxLUCPiKy2dvncDGyJpw9qdE63pnNWSg0Vb0T4ZxFZBPiBo0BwV88lwI9ExAv4gFuMMaesa7cCDwFZwHPWTU0gZ/8AVYVZie6GUmoKiSv4G2O+EKF9M7A5wrU6YHk876vGxqUjf6XUMLrx2wacWrxdKTWMBv9pzhiDy+PVXP5KqSE0+E9zvR4fxmguf6XUUBr8p7lgIRed9lFKhdLgP82druKlwV8pdZoG/2nO2a/BXyk1kgb/ac6lI3+lVBga/Ke5Hp3zV0qFocF/mhss4ahbPZVSITT4T3MuHfkrpcLQ4D/N9eicv1IqDA3+05zL7SXVoSUclVJDaUSY5pz9WsJRKTWSBv9pTnP5K6XC0eA/zTndAxr8lVIjaPCf5lxun2b0VEqNMC7BX0S+IyJGRIpC2u4SkQMisldErgxpXykiO6xr94lORk8ozeWvlAon7uAvIrOBTwLHQtqWAuuBZcA64P5gTV/gAWADgbq+NdZ1NUGcbq+mc1ZKjTAeI/9fAN8FTEjbtcBjxhi3MeYwcABYJSLlQL4x5h1jjAEeBq4bhz6oCFxur875K6VGiCv4i8g1QJMx5qNhlyqBhpCfG622Suv+8PZIr79BROpEpK6trS2ertpWcKunUkqFGjUqiMjLQFmYS3cD3wM+Fe5pYdpMlPawjDEbgY0AtbW1ER+nwguWcMzT4K+UGmbUqGCMuSJcu4icDcwFPrLWbKuAD0RkFYER/eyQh1cBzVZ7VZh2NQH6Bnz4jeb1UUqNdMbTPsaYHcaYEmNMtTGmmkBgP98Y0wo8BawXkQwRmUtgYfc9Y0wL0CMiq61dPjcDW+L/NVQ4wUIuGvyVUsNNSFQwxuwUkU3ALsAL3GaM8VmXbwUeArKA56ybmgBOTeeslIpg3KKCNfoP/fke4J4wj6sDlo/X+6rIBou3p2vwV0oNpSd8p7HB4u068ldKDaPBfxrT4u1KqUimffD/1Z8PsKmuYfQHTkMujwZ/pVR40z74v7CzlS3bmhLdjYRwugNr7LrbRyk13LQP/otK89jb6kx0NxJCp32UUpFM/+Bflke7081JpzvRXZl0LreXFIeQmTbt/zUrpcZo2keFRWV5AOw93pPgnkw+p9tLTnqKlnBUSo1gn+Dfas/gn5eZluhuKKWmoGkf/ItzM5iRncY+O478+73kaC5/pVQY0z74iwiLyvLYY8ORv8ujufyVUuFN++APgR0/+1p7CNSPsQ8t4aiUisQewb8sH5fHR2NHX6K7Mqmc/TryV0qFZ5Pgb89FXy3hqJSKxBbBf2FpLmC/7Z49Ou2jlIrAFsE/LzONysIsW438jTG43F7N5a+UCmtcgr+IfEdEjIgUWT9Xi0ifiGyzbg+GPHaliOwQkQMicp9M0gmkRWV5ttru2T/g1xKOSqmI4g7+IjIb+CRwbNilg8aYFdbtlpD2B4ANBEo71gDr4u1DLBaV5XGwzcmAzz8Zb5dwPe4BQPP6KKXCG4+R/y+A7wKj7qMUkXIg3xjzjgnsu3wYuG4c+jCqxWV5DPgMh9tdk/F2CeeyMnpq8FdKhRNX8BeRa4AmY8xHYS7PFZEPReR1EVljtVUSKPQe1Gi1RXr9DSJSJyJ1bW1t8XSVhaWBHT92OeylxduVUtGMGhlE5GWgLMylu4HvAZ8Kc60FmGOMOSkiK4EnRWQZEG5+P+I3BmPMRmAjQG1tbVwntOYX55LqEPa2dsO5FfG8VFIYLOGowV8pFcaokcEYc0W4dhE5G5gLfGSt2VYBH4jIKmNMK+C2nr9VRA4CCwmM9KtCXqYKaI7rN4hReqqDuUU5tsntr8FfKRXNGU/7GGN2GGNKjDHVxphqAoH9fGNMq4gUi0gKgIjMI7Cwe8gY0wL0iMhqa5fPzcCW+H+N2Cwqy2Pv8e7JeruEcmnxdqVUFBO1z/8SYLuIfAT8EbjFGHPKunYr8BvgAHAQeG6C+jDCotI8Gk71DQbG6Sw48tesnkqpcMZtWGiN/oP3NwObIzyuDlg+Xu87FsE0D/uO93DenBmJ6MKk0WkfpVQ0tjjhG7S4LB/AFoe9XG4vDoGsNB35K6VGslXwr5qRRXZ6ii22e550ecjNSNUSjkqpsGwV/B0OoaY0b9rn+PF4/bxQ38rH589KdFeUUlOUrYI/wKLS3Gk/7fPCzlZOujzc9LGzEt0VpdQUZb/gX5ZPu9NDu9Od6K5MmEfePcrsmVmsWVCU6K4opaYo+wV/K83Dvmk69XOwzclfDp1i/QVzcDh0vl8pFZ79gn/Z9M7x8/t3j5HqEG6orRr9wUop27Jd8C/Oy2BWTvq0nPfvH/Dxxw8auXJZGSV5mYnujlJqCrNd8IdAhs/pOPJ/vr6Vzt4BbvrYnER3RSk1xdky+C8qy2P/8R78/rgShU45j7x7lOpZ2Xx8nm7xVEpFZ9vg7/L4aOrsS3RXxs2+4z28f6SDL63ShV6l1OhsG/xhei36PvruMdJTHFy/Uhd6lVKjs2XwD1b1mi6Lvv0DPv7zg0auXF7GrNyMRHdHKZUEbBn8czNSqZqRNW1G/k9vb6G738tf6UKvUipGtgz+ECjoPl0Oej367lHmF+fwsbkzE90VpVSSsG3wX1iax8E2Jx6vP9Fdicvulm4+ONbJl1bN0QyeSqmYxRX8ReQfRaRJRLZZt0+HXLtLRA6IyF4RuTKkfaWI7LCu3ScJiliLyvLw+g2H212JePtx8+i7x0hP1YVepdTYjMfI/xfGmBXW7VkAEVkKrAeWAeuA+4M1fYEHgA0E6vrWWNcn3ekdP8lb07fX4+XJD5u4+uxyCrPTE90dpVQSmagaf9cCjxlj3MBhETkArBKRI0C+MeYdABF5GLiOSazjGzSvKJdUh/Djp3fxy1cPTPbbj4u+AR89bq+e6FVKjdl4BP/bReRmoA74tjGmA6gE/hLymEarbcC6P7w9LBHZQOBbAnPmjG+AS0918J0rF7G9sXNcX3eyXX1OObVnTe96xEqp8Tdq8BeRl4GyMJfuJjCF82PAWP/8GfB1INw8vonSHpYxZiOwEaC2tnbcczHccun88X5JpZRKCqMGf2PMFbG8kIj8Gnja+rERmB1yuQpottqrwrQrpZSaRPHu9ikP+fFzQL11/ylgvYhkiMhcAgu77xljWoAeEVlt7fK5GdgSTx+UUkqNXbxz/v9XRFYQmLo5AvwNgDFmp4hsAnYBXuA2Y4zPes6twENAFoGF3klf7FVKKbsTY5IjrXFtba2pq6tLdDeUUiqpiMhWY0zt8HbbnvBVSik70+CvlFI2pMFfKaVsSIO/UkrZUNIs+IpIG3D0DJ9eBLSPY3emA/1MRtLPZCT9TMJLps/lLGNM8fDGpAn+8RCRunCr3Xamn8lI+pmMpJ9JeNPhc9FpH6WUsiEN/kopZUN2Cf4bE92BKUg/k5H0MxlJP5Pwkv5zscWcv1JKqaHsMvJXSikVQoO/UkrZ0LQO/iKyziogf0BE7kx0fxJFRP5dRE6ISH1I20wReUlE9lv/tFU5MBGZLSJ/FpHdIrJTRO6w2m37uYhIpoi8JyIfWZ/JP1nttv1MgkQkRUQ+FJGnrZ+T/jOZtsHfKhj/K+AqYCnwJauwvB09BKwb1nYn8IoxpgZ4xfrZTrwEyo4uAVYDt1n/fdj5c3EDnzDGnAusANaJyGrs/ZkE3QHsDvk56T+TaRv8gVXAAWPMIWOMB3iMQGF52zHGvAGcGtZ8LfA76/7vgOsmtVMJZoxpMcZ8YN3vIfA/diU2/lxMgNP6Mc26GWz8mQCISBVwNfCbkOak/0ymc/CvBBpCfo5aLN6GSq3Kalj/LElwfxJGRKqB84B3sfnnYk1vbANOAC8ZY2z/mQD/CnwX8Ie0Jf1nMp2D/5iKxSt7EpFcYDPwTWNMd6L7k2jGGJ8xZgWB+tqrRGR5ovuUSCLyGeCEMWZrovsy3qZz8I9URF4FHA/WYLb+eSLB/Zl0IpJGIPA/Yoz5T6vZ9p8LgDGmE3iNwFqRnT+Ti4BrROQIganjT4jI/2MafCbTOfi/D9SIyFwRSQfWEygsrwKeAr5i3f8KsCWBfZl0IiLAb4Hdxpifh1yy7eciIsUiUmjdzwKuAPZg48/EGHOXMabKGFNNIIa8aoz5MtPgM5nWJ3xF5NME5utSgH83xtyT4C4lhIj8HriMQBra48APgSeBTcAc4BhwgzFm+KLwtCUiFwNvAjs4PZf7PQLz/rb8XETkHAKLlykEBoabjDE/EpFZ2PQzCSUilwHfMcZ8Zjp8JtM6+CullApvOk/7KKWUikCDv1JK2ZAGf6WUsiEN/kopZUMa/JVSyoY0+CullA1p8FdKKRv6/wGTXR/p0xXKswAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "def play_episode(env, agent, max_episode_steps=None, mode=None, render=False):\n",
    "    observation, reward, done = env.reset(), 0., False\n",
    "    agent.reset(mode=mode)\n",
    "    episode_reward, elapsed_steps = 0., 0\n",
    "    while True:\n",
    "        action = agent.step(observation, reward, done)\n",
    "        if render:\n",
    "            env.render()\n",
    "        if done:\n",
    "            break\n",
    "        observation, reward, done, _ = env.step(action)\n",
    "        episode_reward += reward\n",
    "        elapsed_steps += 1\n",
    "        if max_episode_steps and elapsed_steps >= max_episode_steps:\n",
    "            break\n",
    "    agent.close()\n",
    "    return episode_reward, elapsed_steps\n",
    "\n",
    "\n",
    "logging.info('==== train ====')\n",
    "episode_rewards = []\n",
    "for episode in itertools.count():\n",
    "    episode_reward, elapsed_steps = play_episode(env.unwrapped, agent,\n",
    "            max_episode_steps=env._max_episode_steps, mode='train')\n",
    "    episode_rewards.append(episode_reward)\n",
    "    logging.debug('train episode %d: reward = %.2f, steps = %d',\n",
    "            episode, episode_reward, elapsed_steps)\n",
    "    if np.mean(episode_rewards[-10:]) > -120:\n",
    "        break\n",
    "plt.plot(episode_rewards)\n",
    "\n",
    "\n",
    "logging.info('==== test ====')\n",
    "episode_rewards = []\n",
    "for episode in range(100):\n",
    "    episode_reward, elapsed_steps = play_episode(env, agent)\n",
    "    episode_rewards.append(episode_reward)\n",
    "    logging.debug('test episode %d: reward = %.2f, steps = %d',\n",
    "            episode, episode_reward, elapsed_steps)\n",
    "logging.info('average episode reward = %.2f ± %.2f',\n",
    "        np.mean(episode_rewards), np.std(episode_rewards))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "env.close()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
